
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import scrapy
from forex_scraper.items import ForexScraperItem
from selenium import webdriver


class ForexSpider(scrapy.Spider):
    name = "forex"
    start_urls = ["https://www.boc.cn/sourcedb/whpj/"]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        options = webdriver.ChromeOptions()
        options.add_argument("--headless")
        self.driver = webdriver.Chrome(options=options)

    def parse(self, response):
        self.driver.get(response.url)
        try:
            WebDriverWait(self.driver, 10).until(
                EC.presence_of_element_located((By.XPATH, '//table[@width="100%"]'))
            )
            page_source = self.driver.page_source
            response = scrapy.Selector(text=page_source)

            rows = response.xpath('//table[@width="100%"]/tbody/tr')
            for row in rows[1:]:  # Skip the header row
                item = ForexScraperItem()
                # Use default values if any field is missing
                item['currency'] = row.xpath('.//td[1]/text()').get(default="").strip()
                item['tsp'] = row.xpath('.//td[2]/text()').get(default=None)
                item['tbp'] = row.xpath('.//td[3]/text()').get(default=None)
                item['csp'] = row.xpath('.//td[4]/text()').get(default=None)
                item['cbp'] = row.xpath('.//td[5]/text()').get(default=None)
                item['time'] = row.xpath('.//td[8]/text()').get(default=None)

                yield item
        finally:
            self.driver.quit()
